home *** CD-ROM | disk | FTP | other *** search
/ Personal Computer World 2009 February / PCWFEB09.iso / Software / Resources / Chat & Communication / Digsby build 37 / digsby_setup.exe / lib / util / httplib2 / httplib2.pyo (.txt) < prev    next >
Python Compiled Bytecode  |  2008-10-13  |  39KB  |  1,221 lines
  1. # Source Generated with Decompyle++
  2. # File: in.pyo (Python 2.5)
  3.  
  4. from __future__ import with_statement
  5. __author__ = 'Joe Gregorio (joe@bitworking.org)'
  6. __copyright__ = 'Copyright 2006, Joe Gregorio'
  7. __contributors__ = [
  8.     'Thomas Broyer (t.broyer@ltgt.net)',
  9.     'James Antill',
  10.     'Xavier Verges Farrero',
  11.     'Jonathan Feinberg',
  12.     'Blair Zajac',
  13.     'Sam Ruby',
  14.     'Louis Nyffenegger']
  15. __license__ = 'MIT'
  16. __version__ = 'digsby'
  17. import re
  18. import sys
  19. import email
  20. import email.Utils as email
  21. import email.Message as email
  22. import StringIO
  23. import gzip
  24. import zlib
  25. import httplib
  26. import urlparse
  27. import base64
  28. import os
  29. import copy
  30. import calendar
  31. import time
  32. import random
  33. import hmac
  34. from hashlib import md5, sha1
  35. from gettext import gettext as _
  36. import socket
  37. from logging import getLogger
  38. log = getLogger('httplib2')
  39.  
  40. try:
  41.     import socks
  42. except ImportError:
  43.     socks = None
  44.  
  45. if sys.version_info >= (2, 3):
  46.     from iri2uri import iri2uri
  47. else:
  48.     
  49.     def iri2uri(uri):
  50.         return uri
  51.  
  52. __all__ = [
  53.     'Http',
  54.     'Response',
  55.     'ProxyInfo',
  56.     'HttpLib2Error',
  57.     'RedirectMissingLocation',
  58.     'RedirectLimit',
  59.     'FailedToDecompressContent',
  60.     'UnimplementedDigestAuthOptionError',
  61.     'UnimplementedHmacDigestAuthOptionError',
  62.     'debuglevel',
  63.     'cached_content',
  64.     'set_default_proxy',
  65.     'get_default_proxy']
  66. debuglevel = 0
  67. if sys.version_info < (2, 4):
  68.     
  69.     def sorted(seq):
  70.         seq.sort()
  71.         return seq
  72.  
  73.  
  74.  
  75. def HTTPResponse__getheaders(self):
  76.     if self.msg is None:
  77.         raise httplib.ResponseNotReady()
  78.     
  79.     return self.msg.items()
  80.  
  81. if not hasattr(httplib.HTTPResponse, 'getheaders'):
  82.     httplib.HTTPResponse.getheaders = HTTPResponse__getheaders
  83.  
  84.  
  85. class HttpLib2Error(Exception):
  86.     pass
  87.  
  88.  
  89. class HttpLib2ErrorWithResponse(HttpLib2Error):
  90.     
  91.     def __init__(self, desc, response, content):
  92.         self.response = response
  93.         self.content = content
  94.         HttpLib2Error.__init__(self, desc)
  95.  
  96.  
  97.  
  98. class RedirectMissingLocation(HttpLib2ErrorWithResponse):
  99.     pass
  100.  
  101.  
  102. class RedirectLimit(HttpLib2ErrorWithResponse):
  103.     pass
  104.  
  105.  
  106. class FailedToDecompressContent(HttpLib2ErrorWithResponse):
  107.     pass
  108.  
  109.  
  110. class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse):
  111.     pass
  112.  
  113.  
  114. class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse):
  115.     pass
  116.  
  117.  
  118. class RelativeURIError(HttpLib2Error):
  119.     pass
  120.  
  121.  
  122. class ServerNotFoundError(HttpLib2Error):
  123.     pass
  124.  
  125. DEFAULT_MAX_REDIRECTS = 5
  126. HOP_BY_HOP = [
  127.     'connection',
  128.     'keep-alive',
  129.     'proxy-authenticate',
  130.     'proxy-authorization',
  131.     'te',
  132.     'trailers',
  133.     'transfer-encoding',
  134.     'upgrade']
  135.  
  136. def set_default_proxy(proxy_info):
  137.     ProxyInfo.default_proxy_info = proxy_info
  138.  
  139.  
  140. def get_default_proxy():
  141.     return ProxyInfo.default_proxy_info
  142.  
  143.  
  144. def _get_end2end_headers(response):
  145.     hopbyhop = list(HOP_BY_HOP)
  146.     []([ x.strip() for x in response.get('connection', '').split(',') ])
  147.     return _[2]
  148.  
  149. URI = re.compile('^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\\?([^#]*))?(#(.*))?')
  150.  
  151. def parse_uri(uri):
  152.     groups = URI.match(uri).groups()
  153.     return (groups[1], groups[3], groups[4], groups[6], groups[8])
  154.  
  155.  
  156. def urlnorm(uri):
  157.     (scheme, authority, path, query, fragment) = parse_uri(uri)
  158.     if not scheme or not authority:
  159.         raise RelativeURIError('Only absolute URIs are allowed. uri = %s' % uri)
  160.     
  161.     authority = authority.lower()
  162.     scheme = scheme.lower()
  163.     if not path:
  164.         path = '/'
  165.     
  166.     if not query or '?'.join([
  167.         path,
  168.         query]):
  169.         pass
  170.     request_uri = path
  171.     scheme = scheme.lower()
  172.     defrag_uri = scheme + '://' + authority + request_uri
  173.     return (scheme, authority, request_uri, defrag_uri)
  174.  
  175. re_url_scheme = re.compile('^\\w+://')
  176. re_slash = re.compile('[?/:|]+')
  177.  
  178. def safename(filename):
  179.     
  180.     try:
  181.         if re_url_scheme.match(filename):
  182.             if isinstance(filename, str):
  183.                 filename = filename.decode('utf-8')
  184.                 filename = filename.encode('idna')
  185.             else:
  186.                 filename = filename.encode('idna')
  187.     except UnicodeError:
  188.         pass
  189.  
  190.     if isinstance(filename, unicode):
  191.         filename = filename.encode('utf-8')
  192.     
  193.     filemd5 = md5(filename).hexdigest()
  194.     filename = re_url_scheme.sub('', filename)
  195.     filename = re_slash.sub(',', filename)
  196.     if len(filename) > 200:
  197.         filename = filename[:200]
  198.     
  199.     return ','.join((filename, filemd5))
  200.  
  201. NORMALIZE_SPACE = re.compile('(?:\\r\\n)?[ \\t]+')
  202.  
  203. def _normalize_headers(headers):
  204.     return []([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip()) for key, value in headers.iteritems() ])
  205.  
  206.  
  207. def _parse_cache_control(headers):
  208.     retval = { }
  209.     return retval
  210.  
  211. USE_WWW_AUTH_STRICT_PARSING = 0
  212. WWW_AUTH_STRICT = re.compile('^(?:\\s*(?:,\\s*)?([^\\0-\\x1f\\x7f-\\xff()<>@,;:\\\\\\"/[\\]?={} \\t]+)\\s*=\\s*\\"?((?<=\\")(?:[^\\0-\\x08\\x0A-\\x1f\\x7f-\\xff\\\\\\"]|\\\\[\\0-\\x7f])*?(?=\\")|(?<!\\")[^\\0-\\x1f\\x7f-\\xff()<>@,;:\\\\\\"/[\\]?={} \\t]+(?!\\"))\\"?)(.*)$')
  213. WWW_AUTH_RELAXED = re.compile('^(?:\\s*(?:,\\s*)?([^ \\t\\r\\n=]+)\\s*=\\s*\\"?((?<=\\")(?:[^\\\\\\"]|\\\\.)*?(?=\\")|(?<!\\")[^ \\t\\r\\n,]+(?!\\"))\\"?)(.*)$')
  214. UNQUOTE_PAIRS = re.compile('\\\\(.)')
  215.  
  216. def _parse_www_authenticate(headers, headername = 'www-authenticate'):
  217.     retval = { }
  218.     if headers.has_key(headername):
  219.         authenticate = headers[headername].strip()
  220.         if not USE_WWW_AUTH_STRICT_PARSING or WWW_AUTH_STRICT:
  221.             pass
  222.         www_auth = WWW_AUTH_RELAXED
  223.         while authenticate:
  224.             if headername == 'authentication-info':
  225.                 auth_scheme = 'digest'
  226.                 the_rest = authenticate
  227.             else:
  228.                 (auth_scheme, the_rest) = authenticate.split(' ', 1)
  229.             match = www_auth.search(the_rest)
  230.             auth_params = { }
  231.             while match:
  232.                 if match and len(match.groups()) == 3:
  233.                     (key, value, the_rest) = match.groups()
  234.                     auth_params[key.lower()] = UNQUOTE_PAIRS.sub('\\1', value)
  235.                 
  236.                 match = www_auth.search(the_rest)
  237.             retval[auth_scheme.lower()] = auth_params
  238.             authenticate = the_rest.strip()
  239.     
  240.     return retval
  241.  
  242.  
  243. def _entry_disposition(response_headers, request_headers):
  244.     retval = 'STALE'
  245.     cc = _parse_cache_control(request_headers)
  246.     cc_response = _parse_cache_control(response_headers)
  247.     if request_headers.has_key('pragma') and request_headers['pragma'].lower().find('no-cache') != -1:
  248.         retval = 'TRANSPARENT'
  249.         if 'cache-control' not in request_headers:
  250.             request_headers['cache-control'] = 'no-cache'
  251.         
  252.     elif cc.has_key('no-cache'):
  253.         retval = 'TRANSPARENT'
  254.     elif cc_response.has_key('no-cache'):
  255.         retval = 'STALE'
  256.     elif cc.has_key('only-if-cached'):
  257.         retval = 'FRESH'
  258.     elif response_headers.has_key('date'):
  259.         date = calendar.timegm(email.Utils.parsedate_tz(response_headers['date']))
  260.         now = time.time()
  261.         current_age = max(0, now - date)
  262.         if cc_response.has_key('max-age'):
  263.             
  264.             try:
  265.                 freshness_lifetime = int(cc_response['max-age'])
  266.             except ValueError:
  267.                 freshness_lifetime = 0
  268.             except:
  269.                 None<EXCEPTION MATCH>ValueError
  270.             
  271.  
  272.         None<EXCEPTION MATCH>ValueError
  273.         if response_headers.has_key('expires'):
  274.             expires = email.Utils.parsedate_tz(response_headers['expires'])
  275.             if None == expires:
  276.                 freshness_lifetime = 0
  277.             else:
  278.                 freshness_lifetime = max(0, calendar.timegm(expires) - date)
  279.         else:
  280.             freshness_lifetime = 0
  281.         if cc.has_key('max-age'):
  282.             
  283.             try:
  284.                 freshness_lifetime = int(cc['max-age'])
  285.             except ValueError:
  286.                 freshness_lifetime = 0
  287.             except:
  288.                 None<EXCEPTION MATCH>ValueError
  289.             
  290.  
  291.         None<EXCEPTION MATCH>ValueError
  292.         if cc.has_key('min-fresh'):
  293.             
  294.             try:
  295.                 min_fresh = int(cc['min-fresh'])
  296.             except ValueError:
  297.                 min_fresh = 0
  298.  
  299.             current_age += min_fresh
  300.         
  301.         if freshness_lifetime > current_age:
  302.             retval = 'FRESH'
  303.         
  304.     
  305.     return retval
  306.  
  307.  
  308. def _decompressContent(response, new_content):
  309.     content = new_content
  310.     
  311.     try:
  312.         encoding = response.get('content-encoding', None)
  313.         if encoding in ('gzip', 'deflate'):
  314.             if encoding == 'gzip':
  315.                 content = gzip.GzipFile(fileobj = StringIO.StringIO(new_content)).read()
  316.             
  317.             if encoding == 'deflate':
  318.                 content = zlib.decompress(content)
  319.             
  320.             response['content-length'] = str(len(content))
  321.             response['-content-encoding'] = response['content-encoding']
  322.             del response['content-encoding']
  323.     except IOError:
  324.         content = ''
  325.         raise FailedToDecompressContent(_('Content purported to be compressed with %s but failed to decompress.') % response.get('content-encoding'), response, content)
  326.  
  327.     return content
  328.  
  329.  
  330. def _updateCache(request_headers, response_headers, content, cache, cachekey):
  331.     if cachekey:
  332.         cc = _parse_cache_control(request_headers)
  333.         cc_response = _parse_cache_control(response_headers)
  334.         if cc.has_key('no-store') or cc_response.has_key('no-store'):
  335.             cache.delete(cachekey)
  336.         else:
  337.             info = email.Message.Message()
  338.             for key, value in response_headers.iteritems():
  339.                 if key not in ('status', 'content-encoding', 'transfer-encoding'):
  340.                     info[key] = value
  341.                     continue
  342.             
  343.             status = response_headers.status
  344.             if status == 304:
  345.                 status = 200
  346.             
  347.             status_header = 'status: %d\r\n' % response_headers.status
  348.             header_str = info.as_string()
  349.             header_str = re.sub('\r(?!\n)|(?<!\r)\n', '\r\n', header_str)
  350.             text = ''.join([
  351.                 status_header,
  352.                 header_str,
  353.                 content])
  354.             cache.set(cachekey, text, content)
  355.     
  356.  
  357.  
  358. def _cnonce():
  359.     dig = time.ctime()([] % ([], [ '0123456789'[random.randrange(0, 9)] for i in range(20) ])).hexdigest()
  360.     return dig[:16]
  361.  
  362.  
  363. def _wsse_username_token(cnonce, iso_now, password):
  364.     return base64.encodestring(sha1('%s%s%s' % (cnonce, iso_now, password)).digest()).strip()
  365.  
  366.  
  367. class Authentication(object):
  368.     
  369.     def __init__(self, credentials, host, request_uri, headers, response, content, http):
  370.         (scheme, authority, path, query, fragment) = parse_uri(request_uri)
  371.         self.path = path
  372.         self.host = host
  373.         self.credentials = credentials
  374.         self.http = http
  375.  
  376.     
  377.     def depth(self, request_uri):
  378.         (scheme, authority, path, query, fragment) = parse_uri(request_uri)
  379.         return request_uri[len(self.path):].count('/')
  380.  
  381.     
  382.     def inscope(self, host, request_uri):
  383.         (scheme, authority, path, query, fragment) = parse_uri(request_uri)
  384.         if host == self.host:
  385.             pass
  386.         return path.startswith(self.path)
  387.  
  388.     
  389.     def request(self, method, request_uri, headers, content):
  390.         pass
  391.  
  392.     
  393.     def response(self, response, content):
  394.         return False
  395.  
  396.  
  397.  
  398. class BasicAuthentication(Authentication):
  399.     
  400.     def __init__(self, credentials, host, request_uri, headers, response, content, http):
  401.         Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
  402.  
  403.     
  404.     def request(self, method, request_uri, headers, content):
  405.         headers['authorization'] = 'Basic ' + base64.encodestring('%s:%s' % self.credentials).strip()
  406.  
  407.  
  408.  
  409. class DigestAuthentication(Authentication):
  410.     
  411.     def __init__(self, credentials, host, request_uri, headers, response, content, http):
  412.         Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
  413.         challenge = _parse_www_authenticate(response, 'www-authenticate')
  414.         self.challenge = challenge['digest']
  415.         qop = self.challenge.get('qop')
  416.         if not [] in [ x.strip() for x in qop.split() ] or 'auth':
  417.             pass
  418.         self.challenge['qop'] = None
  419.         self.challenge['algorithm'] = self.challenge.get('algorithm', 'MD5')
  420.         if self.challenge['algorithm'] != 'MD5':
  421.             raise UnimplementedDigestAuthOptionError(_('Unsupported value for algorithm: %s.' % self.challenge['algorithm']))
  422.         
  423.         self.A1 = ''.join([
  424.             self.credentials[0],
  425.             ':',
  426.             self.challenge['realm'],
  427.             ':',
  428.             self.credentials[1]])
  429.         self.challenge['nc'] = 1
  430.  
  431.     
  432.     def request(self, method, request_uri, headers, content, cnonce = None):
  433.         
  434.         H = lambda x: md5(x).hexdigest()
  435.         
  436.         KD = lambda s, d: H('%s:%s' % (s, d))
  437.         A2 = ''.join([
  438.             method,
  439.             ':',
  440.             request_uri])
  441.         if not cnonce:
  442.             pass
  443.         self.challenge['cnonce'] = _cnonce()
  444.         request_digest = '"%s"' % KD(H(self.A1), '%s:%s:%s:%s:%s' % (self.challenge['nonce'], '%08x' % self.challenge['nc'], self.challenge['cnonce'], self.challenge['qop'], H(A2)))
  445.         headers['Authorization'] = 'Digest username="%s", realm="%s", nonce="%s", uri="%s", algorithm=%s, response=%s, qop=%s, nc=%08x, cnonce="%s"' % (self.credentials[0], self.challenge['realm'], self.challenge['nonce'], request_uri, self.challenge['algorithm'], request_digest, self.challenge['qop'], self.challenge['nc'], self.challenge['cnonce'])
  446.         self.challenge['nc'] += 1
  447.  
  448.     
  449.     def response(self, response, content):
  450.         if not response.has_key('authentication-info'):
  451.             challenge = _parse_www_authenticate(response, 'www-authenticate').get('digest', { })
  452.             if 'true' == challenge.get('stale'):
  453.                 self.challenge['nonce'] = challenge['nonce']
  454.                 self.challenge['nc'] = 1
  455.                 return True
  456.             
  457.         else:
  458.             updated_challenge = _parse_www_authenticate(response, 'authentication-info').get('digest', { })
  459.             if updated_challenge.has_key('nextnonce'):
  460.                 self.challenge['nonce'] = updated_challenge['nextnonce']
  461.                 self.challenge['nc'] = 1
  462.             
  463.         return False
  464.  
  465.  
  466.  
  467. class HmacDigestAuthentication(Authentication):
  468.     __author__ = 'Thomas Broyer (t.broyer@ltgt.net)'
  469.     
  470.     def __init__(self, credentials, host, request_uri, headers, response, content, http):
  471.         Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
  472.         challenge = _parse_www_authenticate(response, 'www-authenticate')
  473.         self.challenge = challenge['hmacdigest']
  474.         self.challenge['reason'] = self.challenge.get('reason', 'unauthorized')
  475.         if self.challenge['reason'] not in ('unauthorized', 'integrity'):
  476.             self.challenge['reason'] = 'unauthorized'
  477.         
  478.         self.challenge['salt'] = self.challenge.get('salt', '')
  479.         if not self.challenge.get('snonce'):
  480.             raise UnimplementedHmacDigestAuthOptionError(_("The challenge doesn't contain a server nonce, or this one is empty."))
  481.         
  482.         self.challenge['algorithm'] = self.challenge.get('algorithm', 'HMAC-SHA-1')
  483.         if self.challenge['algorithm'] not in ('HMAC-SHA-1', 'HMAC-MD5'):
  484.             raise UnimplementedHmacDigestAuthOptionError(_('Unsupported value for algorithm: %s.' % self.challenge['algorithm']))
  485.         
  486.         self.challenge['pw-algorithm'] = self.challenge.get('pw-algorithm', 'SHA-1')
  487.         if self.challenge['pw-algorithm'] not in ('SHA-1', 'MD5'):
  488.             raise UnimplementedHmacDigestAuthOptionError(_('Unsupported value for pw-algorithm: %s.' % self.challenge['pw-algorithm']))
  489.         
  490.         if self.challenge['algorithm'] == 'HMAC-MD5':
  491.             self.hashmod = md5
  492.         else:
  493.             self.hashmod = sha1
  494.         if self.challenge['pw-algorithm'] == 'MD5':
  495.             self.pwhashmod = md5
  496.         else:
  497.             self.pwhashmod = sha1
  498.         self.key = ''.join([
  499.             self.credentials[0],
  500.             ':',
  501.             self.pwhashmod(''.join([
  502.                 self.credentials[1],
  503.                 self.challenge['salt']])).hexdigest().lower(),
  504.             ':',
  505.             self.challenge['realm']])
  506.         self.key = self.pwhashmod(self.key).hexdigest().lower()
  507.  
  508.     
  509.     def request(self, method, request_uri, headers, content):
  510.         keys = _get_end2end_headers(headers)
  511.         keylist = []([ '%s ' % k for k in keys ])
  512.         headers_val = []([ headers[k] for k in keys ])
  513.         created = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
  514.         cnonce = _cnonce()
  515.         request_digest = '%s:%s:%s:%s:%s' % (method, request_uri, cnonce, self.challenge['snonce'], headers_val)
  516.         request_digest = hmac.new(self.key, request_digest, self.hashmod).hexdigest().lower()
  517.         headers['Authorization'] = 'HMACDigest username="%s", realm="%s", snonce="%s", cnonce="%s", uri="%s", created="%s", response="%s", headers="%s"' % (self.credentials[0], self.challenge['realm'], self.challenge['snonce'], cnonce, request_uri, created, request_digest, keylist)
  518.  
  519.     
  520.     def response(self, response, content):
  521.         challenge = _parse_www_authenticate(response, 'www-authenticate').get('hmacdigest', { })
  522.         if challenge.get('reason') in ('integrity', 'stale'):
  523.             return True
  524.         
  525.         return False
  526.  
  527.  
  528.  
  529. class WsseAuthentication(Authentication):
  530.     
  531.     def __init__(self, credentials, host, request_uri, headers, response, content, http):
  532.         Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
  533.  
  534.     
  535.     def request(self, method, request_uri, headers, content):
  536.         headers['Authorization'] = 'WSSE profile="UsernameToken"'
  537.         iso_now = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
  538.         cnonce = _cnonce()
  539.         password_digest = _wsse_username_token(cnonce, iso_now, self.credentials[1])
  540.         headers['X-WSSE'] = 'UsernameToken Username="%s", PasswordDigest="%s", Nonce="%s", Created="%s"' % (self.credentials[0], password_digest, cnonce, iso_now)
  541.  
  542.  
  543.  
  544. class GoogleLoginAuthentication(Authentication):
  545.     
  546.     def __init__(self, credentials, host, request_uri, headers, response, content, http):
  547.         urlencode = urlencode
  548.         import urllib
  549.         Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
  550.         challenge = _parse_www_authenticate(response, 'www-authenticate')
  551.         service = challenge['googlelogin'].get('service', 'xapi')
  552.         if service == 'xapi' and request_uri.find('calendar') > 0:
  553.             service = 'cl'
  554.         
  555.         auth = dict(Email = credentials[0], Passwd = credentials[1], service = service, source = headers['user-agent'])
  556.         (resp, content) = self.http.request('https://www.google.com/accounts/ClientLogin', method = 'POST', body = urlencode(auth), headers = {
  557.             'Content-Type': 'application/x-www-form-urlencoded' })
  558.         lines = content.split('\n')
  559.         d = [](_[1])
  560.  
  561.     
  562.     def request(self, method, request_uri, headers, content):
  563.         headers['authorization'] = 'GoogleLogin Auth=' + self.Auth
  564.  
  565.  
  566. AUTH_SCHEME_CLASSES = {
  567.     'basic': BasicAuthentication,
  568.     'wsse': WsseAuthentication,
  569.     'digest': DigestAuthentication,
  570.     'hmacdigest': HmacDigestAuthentication,
  571.     'googlelogin': GoogleLoginAuthentication }
  572. AUTH_SCHEME_ORDER = [
  573.     'hmacdigest',
  574.     'googlelogin',
  575.     'digest',
  576.     'wsse',
  577.     'basic']
  578. from path import path
  579. from threading import RLock
  580.  
  581. class ThreadsafeFileCache(object):
  582.     
  583.     def __init__(self, cache):
  584.         self.cache = path(cache)
  585.         self.lock = RLock()
  586.         if not self.cache.exists():
  587.             self.cache.makedirs()
  588.         
  589.  
  590.     
  591.     def get(self, key):
  592.         self.lock.__enter__()
  593.         
  594.         try:
  595.             return (self.cache / safename(key)).bytes()
  596.         except IOError:
  597.             self.lock
  598.             self.lock
  599.         except:
  600.             self.lock
  601.         finally:
  602.             pass
  603.  
  604.  
  605.     
  606.     def getpath(self, key):
  607.         return self.cache / (safename(key) + '.dat')
  608.  
  609.     
  610.     def set(self, key, value, content):
  611.         filepath = self.cache / safename(key)
  612.         self.lock.__enter__()
  613.         
  614.         try:
  615.             filepath.write_bytes(value)
  616.             self.getpath(key).write_bytes(content)
  617.         finally:
  618.             pass
  619.  
  620.  
  621.     
  622.     def delete(self, key):
  623.         filepath = self.cache / safename(key)
  624.         self.lock.__enter__()
  625.         
  626.         try:
  627.             if filepath.exists():
  628.                 filepath.remove()
  629.             
  630.             if contentpath.exists():
  631.                 contentpath.remove()
  632.         finally:
  633.             pass
  634.  
  635.  
  636.     
  637.     def __repr__(self):
  638.         return '<ThreadsafeFileCache %s>' % self.cache
  639.  
  640.  
  641.  
  642. class Credentials(object):
  643.     
  644.     def __init__(self):
  645.         self.credentials = []
  646.  
  647.     
  648.     def add(self, name, password, domain = ''):
  649.         self.credentials.append((domain.lower(), name, password))
  650.  
  651.     
  652.     def clear(self):
  653.         self.credentials = []
  654.  
  655.     
  656.     def iter(self, domain):
  657.         for cdomain, name, password in self.credentials:
  658.             if cdomain == '' or domain == cdomain:
  659.                 yield (name, password)
  660.                 continue
  661.         
  662.  
  663.  
  664.  
  665. class KeyCerts(Credentials):
  666.     pass
  667.  
  668.  
  669. class ProxyInfo(object):
  670.     
  671.     def get_default_proxy():
  672.         return ProxyInfo.default_proxy_info
  673.  
  674.     get_default_proxy = staticmethod(get_default_proxy)
  675.     default_proxy_info = None
  676.     
  677.     def __init__(self, proxy_type, proxy_host, proxy_port, proxy_rdns = None, proxy_user = None, proxy_pass = None):
  678.         (self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, self.proxy_user, self.proxy_pass) = (proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)
  679.  
  680.     
  681.     def astuple(self):
  682.         return (self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, self.proxy_user, self.proxy_pass)
  683.  
  684.     
  685.     def isgood(self):
  686.         if socks and self.proxy_host != None:
  687.             pass
  688.         return self.proxy_port != None
  689.  
  690.  
  691.  
  692. class HTTPConnectionWithTimeout(httplib.HTTPConnection):
  693.     
  694.     def __init__(self, host, port = None, strict = None, timeout = None, proxy_info = None):
  695.         httplib.HTTPConnection.__init__(self, host, port, strict)
  696.         self.timeout = timeout
  697.         if not proxy_info:
  698.             pass
  699.         self.proxy_info = ProxyInfo.get_default_proxy()
  700.  
  701.     
  702.     def connect(self):
  703.         msg = 'getaddrinfo returns an empty list'
  704.         for res in socket.getaddrinfo(self.host, self.port, 0, socket.SOCK_STREAM):
  705.             (af, socktype, proto, canonname, sa) = res
  706.             
  707.             try:
  708.                 if self.proxy_info and self.proxy_info.isgood():
  709.                     if self.proxy_info.proxy_type == socks.PROXY_TYPE_HTTP:
  710.                         self.sock = socket.socket(af, socktype, proto)
  711.                         self._orig_host = self.host
  712.                         self._orig_port = self.port
  713.                         self.host = self.proxy_info.proxy_host
  714.                         self.port = self.proxy_info.proxy_port
  715.                     else:
  716.                         self.sock = socks.socksocket(af, socktype, proto)
  717.                         self.sock.setproxy(*self.proxy_info.astuple())
  718.                 else:
  719.                     self.sock = socket.socket(af, socktype, proto)
  720.                 if self.timeout is not None:
  721.                     self.sock.settimeout(self.timeout)
  722.                 
  723.                 if self.debuglevel > 0:
  724.                     print 'connect: (%s, %s)' % (self.host, self.port)
  725.                 
  726.                 self.sock.connect(sa)
  727.             except socket.error:
  728.                 msg = None
  729.                 if self.debuglevel > 0:
  730.                     print 'connect fail:', (self.host, self.port), msg
  731.                 
  732.                 if self.sock:
  733.                     self.sock.close()
  734.                 
  735.                 self.sock = None
  736.                 continue
  737.  
  738.             break
  739.         
  740.         if not self.sock:
  741.             raise socket.error, msg
  742.         
  743.  
  744.  
  745. if hasattr(httplib, 'HTTPSConnection'):
  746.     
  747.     class HTTPSConnectionWithTimeout(httplib.HTTPSConnection):
  748.         
  749.         def __init__(self, host, port = None, key_file = None, cert_file = None, strict = None, timeout = None, proxy_info = None):
  750.             self.timeout = timeout
  751.             self.proxy_info = proxy_info
  752.             httplib.HTTPSConnection.__init__(self, host, port = port, key_file = key_file, cert_file = cert_file, strict = strict)
  753.  
  754.         
  755.         def connect(self):
  756.             if self.proxy_info and self.proxy_info.isgood():
  757.                 sock = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM)
  758.                 sock.setproxy(*self.proxy_info.astuple())
  759.             else:
  760.                 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  761.             if self.timeout is not None:
  762.                 sock.settimeout(self.timeout)
  763.             
  764.             sock.connect((self.host, self.port))
  765.             ssl = socket.ssl(sock, self.key_file, self.cert_file)
  766.             self.sock = httplib.FakeSocket(sock, ssl)
  767.  
  768.  
  769.  
  770. redirect_responses = set([
  771.     300,
  772.     301,
  773.     302,
  774.     303,
  775.     307])
  776.  
  777. class Http(object):
  778.     
  779.     def __init__(self, cache = None, timeout = None, proxy_info = None):
  780.         if not proxy_info:
  781.             pass
  782.         self.proxy_info = ProxyInfo.get_default_proxy()
  783.         self.connections = { }
  784.         if cache and isinstance(cache, basestring):
  785.             self.cache = ThreadsafeFileCache(cache)
  786.         else:
  787.             self.cache = cache
  788.         self.credentials = Credentials()
  789.         self.certificates = KeyCerts()
  790.         self.authorizations = []
  791.         self.follow_redirects = True
  792.         self.follow_all_redirects = False
  793.         self.ignore_etag = False
  794.         self.force_exception_to_status_code = False
  795.         self.timeout = timeout
  796.  
  797.     
  798.     def _auth_from_challenge(self, host, request_uri, headers, response, content):
  799.         challenges = _parse_www_authenticate(response, 'www-authenticate')
  800.         for cred in self.credentials.iter(host):
  801.             for scheme in AUTH_SCHEME_ORDER:
  802.                 if challenges.has_key(scheme):
  803.                     yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, headers, response, content, self)
  804.                     continue
  805.             
  806.         
  807.  
  808.     
  809.     def add_credentials(self, name, password, domain = ''):
  810.         self.credentials.add(name, password, domain)
  811.  
  812.     
  813.     def add_certificate(self, key, cert, domain):
  814.         self.certificates.add(key, cert, domain)
  815.  
  816.     
  817.     def clear_credentials(self):
  818.         self.credentials.clear()
  819.         self.authorizations = []
  820.  
  821.     
  822.     def _conn_request(self, conn, request_uri, method, body, headers):
  823.         for i in range(2):
  824.             
  825.             try:
  826.                 conn.request(method, request_uri, body, headers)
  827.                 response = conn.getresponse()
  828.             except socket.gaierror:
  829.                 conn.close()
  830.                 raise ServerNotFoundError('Unable to find the server at %s' % conn.host)
  831.             except (socket.error, httplib.HTTPException):
  832.                 if i == 0:
  833.                     conn.close()
  834.                     conn.connect()
  835.                     continue
  836.                 else:
  837.                     raise 
  838.             except:
  839.                 i == 0
  840.  
  841.             content = response.read()
  842.             response = Response(response)
  843.             if method != 'HEAD':
  844.                 content = _decompressContent(response, content)
  845.             
  846.             break
  847.         
  848.         return (response, content)
  849.  
  850.     
  851.     def _request(self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey):
  852.         auths = _[1]
  853.         if not auths or sorted(auths)[0][1]:
  854.             pass
  855.         auth = None
  856.         (response, content) = self._conn_request(conn, request_uri, method, body, headers)
  857.         if auth:
  858.             if auth.response(response, body):
  859.                 auth.request(method, request_uri, headers, body)
  860.                 (response, content) = self._conn_request(conn, request_uri, method, body, headers)
  861.                 response._stale_digest = 1
  862.             
  863.         
  864.         if response.status == 401:
  865.             for authorization in self._auth_from_challenge(host, request_uri, headers, response, content):
  866.                 authorization.request(method, request_uri, headers, body)
  867.                 (response, content) = self._conn_request(conn, request_uri, method, body, headers)
  868.                 if response.status != 401:
  869.                     self.authorizations.append(authorization)
  870.                     authorization.response(response, body)
  871.                     break
  872.                     continue
  873.             
  874.         
  875.         if self.follow_all_redirects and method in ('GET', 'HEAD') or response.status == 303:
  876.             if self.follow_redirects and response.status in redirect_responses:
  877.                 if redirections:
  878.                     if not response.has_key('location') and response.status != 300:
  879.                         raise RedirectMissingLocation(_('Redirected but the response is missing a Location: header.'), response, content)
  880.                     
  881.                     if response.has_key('location'):
  882.                         location = response['location']
  883.                         (scheme, authority, path, query, fragment) = parse_uri(location)
  884.                         if authority == None:
  885.                             response['location'] = urlparse.urljoin(absolute_uri, location)
  886.                         
  887.                     
  888.                     if response.status == 301 and method in ('GET', 'HEAD'):
  889.                         response['-x-permanent-redirect-url'] = response['location']
  890.                         if not response.has_key('content-location'):
  891.                             response['content-location'] = absolute_uri
  892.                         
  893.                         _updateCache(headers, response, content, self.cache, cachekey)
  894.                         if cachekey:
  895.                             response.cache_path = self.cache.getpath(cachekey)
  896.                         
  897.                     
  898.                     if headers.has_key('if-none-match'):
  899.                         del headers['if-none-match']
  900.                     
  901.                     if headers.has_key('if-modified-since'):
  902.                         del headers['if-modified-since']
  903.                     
  904.                     if response.has_key('location'):
  905.                         location = response['location']
  906.                         old_response = copy.deepcopy(response)
  907.                         if not old_response.has_key('content-location'):
  908.                             old_response['content-location'] = absolute_uri
  909.                         
  910.                         if not response.status == 303 or method not in ('GET', 'HEAD') or 'GET':
  911.                             pass
  912.                         redirect_method = method
  913.                         (response, content) = self.request(location, redirect_method, body = body, headers = headers, redirections = redirections - 1)
  914.                         response.previous = old_response
  915.                     
  916.                 else:
  917.                     raise RedirectLimit(_('Redirected more times than rediection_limit allows.'), response, content)
  918.             elif response.status in (200, 203) and method == 'GET':
  919.                 if not response.has_key('content-location'):
  920.                     response['content-location'] = absolute_uri
  921.                 
  922.                 _updateCache(headers, response, content, self.cache, cachekey)
  923.                 if cachekey:
  924.                     response.cache_path = self.cache.getpath(cachekey)
  925.                 
  926.             
  927.         
  928.         return (response, content)
  929.  
  930.     
  931.     def request(self, uri, method = 'GET', body = None, headers = None, redirections = DEFAULT_MAX_REDIRECTS, connection_type = None, decode = True):
  932.         
  933.         try:
  934.             if headers is None:
  935.                 headers = { }
  936.             else:
  937.                 headers = _normalize_headers(headers)
  938.             if not headers.has_key('user-agent'):
  939.                 headers['user-agent'] = 'Python-httplib2/%s' % __version__
  940.             
  941.             uri = iri2uri(uri)
  942.             (scheme, authority, request_uri, defrag_uri) = urlnorm(uri)
  943.             domain_port = authority.split(':')[0:2]
  944.             if len(domain_port) == 2 and domain_port[1] == '443' and scheme == 'http':
  945.                 scheme = 'https'
  946.                 authority = domain_port[0]
  947.             
  948.             if scheme == 'http' and self.proxy_info and self.proxy_info.isgood() and self.proxy_info.proxy_type == socks.PROXY_TYPE_HTTP:
  949.                 if not request_uri:
  950.                     slash = '/'
  951.                 else:
  952.                     slash = ''
  953.                 request_uri = uri + slash
  954.                 user = self.proxy_info.proxy_user
  955.                 password = self.proxy_info.proxy_pass
  956.                 if user and password:
  957.                     unquote = unquote
  958.                     import urllib
  959.                     user_pass = '%s:%s' % (unquote(user), unquote(password))
  960.                     creds = base64.b64encode(user_pass).strip()
  961.                     headers['Proxy-Authorization'] = 'Basic ' + creds
  962.                 
  963.                 newhost = self.proxy_info.proxy_host
  964.                 newport = self.proxy_info.proxy_port
  965.                 use_port = newport != httplib.HTTP_PORT
  966.                 if use_port:
  967.                     authority = newhost + ':' + str(newport)
  968.                 else:
  969.                     authority = newhost
  970.             
  971.             conn_key = scheme + ':' + authority
  972.             if conn_key in self.connections:
  973.                 conn = self.connections[conn_key]
  974.             elif not connection_type:
  975.                 if not scheme == 'https' or HTTPSConnectionWithTimeout:
  976.                     pass
  977.                 connection_type = HTTPConnectionWithTimeout
  978.             
  979.             certs = list(self.certificates.iter(authority))
  980.             if scheme == 'https' and certs:
  981.                 conn = self.connections[conn_key] = connection_type(authority, key_file = certs[0][0], cert_file = certs[0][1], timeout = self.timeout, proxy_info = self.proxy_info)
  982.             else:
  983.                 conn = self.connections[conn_key] = connection_type(authority, timeout = self.timeout, proxy_info = self.proxy_info)
  984.             conn.set_debuglevel(debuglevel)
  985.             if method in ('GET', 'HEAD') and 'range' not in headers:
  986.                 headers['accept-encoding'] = 'compress, gzip'
  987.             
  988.             info = email.Message.Message()
  989.             cached_value = None
  990.             if self.cache:
  991.                 cachekey = defrag_uri
  992.                 cached_value = self.cache.get(cachekey)
  993.                 if cached_value:
  994.                     info = email.message_from_string(cached_value)
  995.                     cache_path = self.cache.getpath(cachekey)
  996.                     
  997.                     try:
  998.                         content = cached_value.split('\r\n\r\n', 1)[1]
  999.                     except IndexError:
  1000.                         self.cache.delete(cachekey)
  1001.                         cachekey = None
  1002.                         cached_value = None
  1003.                         cache_path = None
  1004.                     except:
  1005.                         None<EXCEPTION MATCH>IndexError
  1006.                     
  1007.  
  1008.                 None<EXCEPTION MATCH>IndexError
  1009.             else:
  1010.                 cachekey = None
  1011.             if method in ('PUT',) and self.cache and info.has_key('etag') and not (self.ignore_etag) and 'if-match' not in headers:
  1012.                 headers['if-match'] = info['etag']
  1013.             
  1014.             if method not in ('GET', 'HEAD') and self.cache and cachekey:
  1015.                 self.cache.delete(cachekey)
  1016.             
  1017.             if cached_value and method in ('GET', 'HEAD') and self.cache and 'range' not in headers:
  1018.                 if info.has_key('-x-permanent-redirect-url'):
  1019.                     (response, new_content) = self.request(info['-x-permanent-redirect-url'], 'GET', headers = headers, redirections = redirections - 1)
  1020.                     response.previous = Response(info)
  1021.                     response.previous.fromcache = True
  1022.                 else:
  1023.                     entry_disposition = _entry_disposition(info, headers)
  1024.                     if entry_disposition == 'FRESH':
  1025.                         if not cached_value:
  1026.                             info['status'] = '504'
  1027.                             content = ''
  1028.                         
  1029.                         response = Response(info)
  1030.                         if cached_value:
  1031.                             response.fromcache = True
  1032.                             response.cache_path = cache_path
  1033.                         
  1034.                         return (response, content)
  1035.                     
  1036.                     if entry_disposition == 'STALE':
  1037.                         if info.has_key('etag') and not (self.ignore_etag) and 'if-none-match' not in headers:
  1038.                             headers['if-none-match'] = info['etag']
  1039.                         
  1040.                         if info.has_key('last-modified') and 'last-modified' not in headers:
  1041.                             headers['if-modified-since'] = info['last-modified']
  1042.                         
  1043.                     elif entry_disposition == 'TRANSPARENT':
  1044.                         pass
  1045.                     
  1046.                     (response, new_content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
  1047.                 if response.status == 304 and method == 'GET':
  1048.                     for key in _get_end2end_headers(response):
  1049.                         info[key] = response[key]
  1050.                     
  1051.                     merged_response = Response(info)
  1052.                     if hasattr(response, '_stale_digest'):
  1053.                         merged_response._stale_digest = response._stale_digest
  1054.                     
  1055.                     _updateCache(headers, merged_response, content, self.cache, cachekey)
  1056.                     response = merged_response
  1057.                     response.status = 200
  1058.                     response.fromcache = True
  1059.                     response.cache_path = cache_path
  1060.                 elif response.status == 200:
  1061.                     content = new_content
  1062.                 else:
  1063.                     self.cache.delete(cachekey)
  1064.                     content = new_content
  1065.             else:
  1066.                 (response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
  1067.         except Exception:
  1068.             e = None
  1069.             if self.force_exception_to_status_code:
  1070.                 if isinstance(e, HttpLib2ErrorWithResponse):
  1071.                     response = e.response
  1072.                     content = e.content
  1073.                     response.status = 500
  1074.                     response.reason = str(e)
  1075.                 elif isinstance(e, socket.timeout):
  1076.                     content = 'Request Timeout'
  1077.                     response = Response({
  1078.                         'content-type': 'text/plain',
  1079.                         'status': '408',
  1080.                         'content-length': len(content) })
  1081.                     response.reason = 'Request Timeout'
  1082.                 else:
  1083.                     content = str(e)
  1084.                     response = Response({
  1085.                         'content-type': 'text/plain',
  1086.                         'status': '400',
  1087.                         'content-length': len(content) })
  1088.                     response.reason = 'Bad Request'
  1089.             else:
  1090.                 raise 
  1091.         except:
  1092.             self.force_exception_to_status_code
  1093.  
  1094.         if decode:
  1095.             ctype_data = response.get('content-type', None)
  1096.             if ctype_data is not None:
  1097.                 while ctype_data:
  1098.                     split = ctype_data.split(';', 1)
  1099.                     if len(split) == 1:
  1100.                         split = [
  1101.                             split[0],
  1102.                             '']
  1103.                     
  1104.                     (param, ctype_data) = split
  1105.                     param = param.strip()
  1106.                     if '=' in param:
  1107.                         (key, val) = param.split('=', 1)
  1108.                         if key.lower() == 'charset':
  1109.                             charset = val
  1110.                             
  1111.                             try:
  1112.                                 content = content.decode(charset)
  1113.                             except:
  1114.                                 pass
  1115.  
  1116.                             ctype_data = None
  1117.                             continue
  1118.                         
  1119.                     key.lower() == 'charset'
  1120.             
  1121.         
  1122.         return (response, content)
  1123.  
  1124.  
  1125. HTTP = Http
  1126.  
  1127. class Response(dict):
  1128.     fromcache = False
  1129.     version = 11
  1130.     status = 200
  1131.     reason = 'Ok'
  1132.     previous = None
  1133.     
  1134.     def __init__(self, info):
  1135.         if isinstance(info, httplib.HTTPResponse):
  1136.             for key, value in info.getheaders():
  1137.                 self[key] = value
  1138.             
  1139.             self.status = info.status
  1140.             self['status'] = str(self.status)
  1141.             self.reason = info.reason
  1142.             self.version = info.version
  1143.         elif isinstance(info, email.Message.Message):
  1144.             for key, value in info.items():
  1145.                 self[key] = value
  1146.             
  1147.             self.status = int(self['status'])
  1148.         else:
  1149.             for key, value in info.iteritems():
  1150.                 self[key] = value
  1151.             
  1152.             self.status = int(self.get('status', self.status))
  1153.  
  1154.     
  1155.     def ok(self):
  1156.         return self.status // 100 == 2
  1157.  
  1158.     ok = property(ok)
  1159.     
  1160.     def __getattr__(self, name):
  1161.         if name == 'dict':
  1162.             return self
  1163.         else:
  1164.             raise AttributeError, name
  1165.  
  1166.  
  1167.  
  1168. def cached_content(cache_path):
  1169.     log.info('httplib2.cached_content(%s)' % cache_path)
  1170.     
  1171.     try:
  1172.         
  1173.         try:
  1174.             f = _[2]
  1175.             cached_value = f.read()
  1176.         finally:
  1177.             pass
  1178.  
  1179.         log.info('read %d bytes of data: %r', len(cached_value), cached_value[:40])
  1180.     except Exception:
  1181.         e = None
  1182.         log.error('error loading cached data from %s', cache_path)
  1183.         return None
  1184.  
  1185.     
  1186.     try:
  1187.         data = cached_value.split('\r\n\r\n', 1)[1]
  1188.         log.info('returning %d bytes of data', len(data))
  1189.         return data
  1190.     except IndexError:
  1191.         log.error('index error, returning none')
  1192.         return None
  1193.  
  1194.  
  1195.  
  1196. def main():
  1197.     import wx as wx
  1198.     import PIL
  1199.     import cStringIO
  1200.     import gui.toolbox as gui
  1201.     
  1202.     def show(data):
  1203.         PIL.Image.open(cStringIO.StringIO(data)).Show()
  1204.  
  1205.     a = wx.PySimpleApp()
  1206.     f = wx.Frame(None)
  1207.     f.Show()
  1208.     url = 'http://www.google.com/intl/en_ALL/images/logo.gif'
  1209.     (resp, data) = Http('c:\\test_cache').request(url)
  1210.     print resp
  1211.     if resp.status <= resp.status:
  1212.         pass
  1213.     elif resp.status < 300:
  1214.         show(data)
  1215.     
  1216.     a.MainLoop()
  1217.  
  1218. if __name__ == '__main__':
  1219.     main()
  1220.  
  1221.